package main;

import utils.FileUtil;
import utils.HttpConnectionUtil;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLConnection;
import java.util.ArrayList;

import static spider.HtmlLoder.UrlConnIsReadToString;
import static spider.HtmlLoder.getValuableByRegex;

/**
 * @author sunlggggg
 * @date 2017/7/25
 */
public class FcwDownload {
    public static void main(String[] args) {
        FileWriter fw = FileUtil.getFileWriter("d:/124.txt", true);
        BufferedWriter bw = FileUtil.getBufferWriter(fw);
        ArrayList<String> arrayList = new ArrayList();
        for (int i = 6721 ; i < 9780; i++) {
            String url_str = "http://fcw18.com/embed/" + i;
            URLConnection urlConnection = HttpConnectionUtil.getHttpConnection(url_str);
            InputStream htm_in = null;
            try {
                htm_in = urlConnection.getInputStream();
            } catch (IOException e) {
                System.out.println(url_str + " 404 not found");
                //e.printStackTrace();
            }
            if (htm_in == null) {
                continue;
            }
            String htm_str = null;
            try {
                htm_str = UrlConnIsReadToString(htm_in, "UTF-8");
            } catch (IOException e) {
                e.printStackTrace();
            }
            System.out.println(url_str);
            for (String s : getValuableByRegex(htm_str, "<title>(.*)/ Embed Player", 1))
                arrayList.add(url_str + "\t\t" + s);
            if(i%20 == 0 ){
                FileUtil.WriteToFile(bw, arrayList, true);
                arrayList.clear();
            }
        }

        FileUtil.Close(bw);
        FileUtil.Close(fw);
    }
}
